package com.thinkgem.jeesite.modules.spider.processor;

import java.util.List;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.thinkgem.jeesite.modules.spider.entity.StackoverflowUser;
import com.virjar.dungproxy.client.webmagic.DungProxyDownloader;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;

public class StackoverflowUserProcessor implements PageProcessor {
	private Logger logger = LoggerFactory.getLogger(getClass());
	private Site site;

	@Override
	public Site getSite() {
		site = Site.me().setRetryTimes(3) // 就我的经验,这个重试一般用处不大,他是httpclient内部重试
				.setTimeOut(30000)// 在使用代理的情况下,这个需要设置,可以考虑调大线程数目
				.setSleepTime(0)// 使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
				.setCycleRetryTimes(3)// 这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
				.setUseGzip(true)
				// .setRetryTimes(3).setRetrySleepTime(3000).setSleepTime(1000).setTimeOut(8000)
				.addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
				.addHeader("Accept-Encoding", "gzip, deflate, br")
				.addHeader("Accept-Language", "zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3")
				.addHeader("Cache-Control", "no-cache").addHeader("Connection", "keep-alive")
				.addHeader("Pragma", "no-cache").addHeader("Upgrade-Insecure-Requests", "1").addHeader("User-Agent",
						"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:49.0) Gecko/20100101 Firefox/49.0");
		return site;
	}

	private int parseStringValue(String str) {
		int ret = 0;
		if (str == null) {
			str = "";
		}
		str.replace(",", "").trim();
		try {
			ret = Integer.parseInt(str);
		} catch (NumberFormatException e) {
			// do nothing
		}
		return ret;
	}

	@Override
	public void process(Page page) {
		Html html = page.getHtml();
		String url = page.getUrl().toString();
		logger.debug("processing url==========>" + url);
		// System.out.println(html.toString());

		List<String> detailInfoList = html.xpath("//div[@class='user-details']/a/@href").all();
		if (detailInfoList.size() != 0) {
			List<String> pageList = html.xpath("//div[@class='pager fr']/a/@href").all();
			for (String pageUrl : pageList) {
				page.addTargetRequest(pageUrl);
			}
			for (String userUrl : detailInfoList) {
				page.addTargetRequest(userUrl);
			}
		} else {
			String name = html.xpath("//h2[@class='user-card-name']/text()").toString();
			String introduction = html.xpath("//div[@class='bio']").toString();
			String reputation = html.xpath("//div[@class='reputation']/text()").toString();
			String gold = html.xpath("//span[@class='badge1-alternate']/span[@class='badgecount']/text()").toString();
			String silver = html.xpath("//span[@class='badge2-alternate']/span[@class='badgecount']/text()").toString();
			String bronze = html.xpath("//span[@class='badge3-alternate']/span[@class='badgecount']/text()").toString();
			String answer = html.xpath("//div[@class='stat answers col-3']/span[@class='number']/text()").toString();
			String question = html.xpath("//div[@class='stat questions col-3']/span[@class='number']/text()")
					.toString();
			String reached = html.xpath("//div[@class='stat people-helped col-5']/span[@class='number']/text()")
					.toString();

			StackoverflowUser user = new StackoverflowUser();
			user.setUsername(name);
			user.setSelfintroduction(introduction);
			user.setReputation(parseStringValue(reputation));
			user.setGold(parseStringValue(gold));
			user.setSilver(parseStringValue(silver));
			user.setBronze(parseStringValue(bronze));
			user.setAnswer(parseStringValue(answer));
			user.setQuestion(parseStringValue(question));
			user.setReached(reached);
			user.setSourceUrl(url);

			page.putField("user", user);
		}
	}

	public static void main(String[] args) {
		Spider.create(new StackoverflowUserProcessor()).setDownloader(new DungProxyDownloader()).thread(500)
				.addUrl("http://stackoverflow.com/users").run();
	}
}
